/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package com.haipv.qa.dataset.yahoo.answers;

import java.io.*;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.ProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.RedirectStrategy;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.apache.http.protocol.HttpContext;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import thesis.haipv.qa.model.UserProfile;
import thesis.haipv.qa.persistence.AnswerDAO;
import thesis.haipv.qa.persistence.FeedbackDAO;
import thesis.haipv.qa.persistence.QuestionDAO;
import thesis.haipv.qa.persistence.UserProfileDAO;
import thesis.haipv.qa.utils.Util;

/**
 *
 * @author PHANHAI
 */
public class CrawlerYahooAnswerAPI {

    private static final String API_KEY = "HaPyMo6k";
    private static final String USER_AGENT = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.14) Gecko/20080404 Firefox/2.0.0.14";
    public long category_id;
    private String url = "";
    //"http://answers.yahooapis.com/AnswersService/V1/getByCategory?appid=HaPyMo6k&category_id=396545664&type=resolved&region=vn&results=50&start=1&output=json";

    public CrawlerYahooAnswerAPI() {
        category_id = 396545660;
    }

    public void sendRequest(int start) throws IOException {
        long _start = 1 + 50 * start;
        url = "http://answers.yahooapis.com/AnswersService/V1/getByCategory?appid=" + API_KEY
                + "&output=json&type=resolved&region=vn&results=50&category_id=" + category_id + "&start=" + _start;
        HttpClient client = new DefaultHttpClient();
        HttpGet request = new HttpGet(url);

        // add request header
        request.addHeader("User-Agent", USER_AGENT);

        HttpResponse response = client.execute(request);

        System.out.println("\nSending 'GET' request to URL : " + url);
        System.out.println("Response Code : "
                + response.getStatusLine().getStatusCode());

        BufferedReader rd = new BufferedReader(
                new InputStreamReader(response.getEntity().getContent()));

        StringBuffer result = new StringBuffer();
        String line = "";
        while ((line = rd.readLine()) != null) {
            result.append(line);
        }
        System.out.println("Crawler successful!");
        String filename = "dataset/yh" + category_id + "_" + _start;
        writeData(filename, result.toString());
        System.out.println("write data successful!");

    }

    public static boolean check(String id) {
        //long _start = 1 + 50 * start;

        DefaultHttpClient client = new DefaultHttpClient();
        String url = "http://vn.answers.yahoo.com/question/index?qid=" + id;
        HttpGet request = new HttpGet(url);
        boolean ok = false;
        try {
            // add request header
            request.addHeader("User-Agent", USER_AGENT);

            HttpResponse response = client.execute(request);

            BufferedReader rd = new BufferedReader(
                    new InputStreamReader(response.getEntity().getContent()));

            StringBuffer result = new StringBuffer();
            String line = "";
            while ((line = rd.readLine()) != null) {
                result.append(line);
            }

            ok = result.toString().contains("Do người đặt câu hỏi bình chọn");
        } catch (Exception e) {
            e.printStackTrace();
        }
        return ok;
    }

    public static void sendRequestQuestion(ArrayList<String> ids) {
        try {
            HttpClient client = new DefaultHttpClient();
            long category = 396545660;
            long index = 0;
            System.out.println("Writing....");
            for (String id : ids) {
                if (index < 1002) {
                    ++index;
                    continue;
                }
                String url = "http://answers.yahooapis.com/AnswersService/V1/getQuestion?appid=" + API_KEY
                        + "&question_id=" + id + "&output=json";

                HttpGet request = new HttpGet(url);

                // add request header
                request.addHeader("User-Agent", USER_AGENT);

                HttpResponse response = client.execute(request);

                BufferedReader rd = new BufferedReader(
                        new InputStreamReader(response.getEntity().getContent()));

                StringBuffer result = new StringBuffer();
                String line = "";
                while ((line = rd.readLine()) != null) {
                    result.append(line);
                }
                String filename = "dataset/yh_computer_internet_q" + index;
                writeData(filename, result.toString());
                ++index;
            }
            System.out.println("write data successful!");
        } catch (Exception ex) {
        }
    }

    private static void writeData(String filename, String content) {
        File file = new File(filename);
        if (!file.exists()) {
            try {
                file.createNewFile();
            } catch (Exception ex) {
                ex.printStackTrace();
            }
        }
        try {
            Writer out = new BufferedWriter(new OutputStreamWriter(
                    new FileOutputStream(file), "UTF8"));
            out.write(content);
            out.close();
        } catch (Exception ex) {
            ex.printStackTrace();
        }
    }

    public static void readJson() {
        try {
            JSONParser parser = new JSONParser();
            JSONObject jsonRoot = (JSONObject) parser.parse(new BufferedReader(new FileReader("dataset/yh396545660_1")));
            JSONObject jsonQuestions = (JSONObject) jsonRoot.get("all");

            JSONArray jsonArrQuestions = (JSONArray) jsonQuestions.get("questions");
            //System.out.println(jsonArrQuestions.size());
            for (int i = 0; i < 50; ++i) {
                JSONObject json = (JSONObject) jsonArrQuestions.get(i);
                String count = (String) json.get("Subject");
                System.out.println(count);
            }
        } catch (Exception ex) {
            ex.printStackTrace();
        }
    }

    public static void writeIdQuestion() throws FileNotFoundException, IOException {
        int f;
        File file = new File("dataset/computer_internet/qid1");
        if (!file.exists()) {
            file.createNewFile();
        }
        Writer out = new BufferedWriter(new OutputStreamWriter(
                new FileOutputStream(file)));
        JSONParser parser = new JSONParser();
        StringBuffer result = new StringBuffer();
        for (int k = 0; k <= 40; ++k) {
            f = 1 + 50 * k;
            try {
                String fname = "dataset/computer_internet/yh396545660_" + f;
                //System.out.println("FILE: " + fname);
                JSONObject jsonRoot = (JSONObject) parser.parse(new BufferedReader(new FileReader(fname)));
                JSONObject jsonQuestions = (JSONObject) jsonRoot.get("all");

                JSONArray jsonArrQuestions = (JSONArray) jsonQuestions.get("questions");
                //  System.out.println(jsonArrQuestions.size());
                for (int i = 0; i < jsonArrQuestions.size(); ++i) {
                    JSONObject json = (JSONObject) jsonArrQuestions.get(i);
                    String id = (String) json.get("Id");
                    System.out.println(id);
                    result.append(id + "\n");
                }
            } catch (Exception ex) {
                ex.printStackTrace();
            }
        }
        out.write(result.toString());
    }

    public static ArrayList<String> readId() {
        FileInputStream fis = null;
        ArrayList<String> ids = new ArrayList<>();
        try {
            String fname = "dataset/computer_internet/qid1";
            fis = new FileInputStream(fname);
            DataInputStream dis = new DataInputStream(fis);

            String line = "";
            while ((line = dis.readLine()) != null) {
                ids.add(line);
            }
        } catch (Exception ex) {
            Logger.getLogger(CrawlerYahooAnswerAPI.class.getName()).log(Level.SEVERE, null, ex);
        } finally {
            try {
                fis.close();
            } catch (IOException ex) {
                Logger.getLogger(CrawlerYahooAnswerAPI.class.getName()).log(Level.SEVERE, null, ex);
            }
        }
        return ids;
    }

    public static void createQuestion() {
        String f = "dataset/yh_computer_internet_q";
        JSONParser parser = new JSONParser();
        SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        System.out.println("Running....");
        for (int k = 1002; k < 2005; ++k) {
            try {
                String fname = f + k;
                //System.out.println("FILE: " + fname);
                JSONObject jsonRoot = (JSONObject) parser.parse(new BufferedReader(new FileReader(fname)));

                JSONObject jsonQuestions = (JSONObject) jsonRoot.get("all");

                JSONArray jsonArrQuestions = (JSONArray) jsonQuestions.get("question");
                JSONObject json = (JSONObject) jsonArrQuestions.get(0);

                String id = (String) json.get("Id");
                String subject = (String) json.get("Subject");
                String content = (String) json.get("Content");
                String date = (String) json.get("Date");
                Date d = df.parse(date);
                Calendar gc = new GregorianCalendar();
                gc.setTime(d);
                long category = (long) json.get("CategoryId");
                String userId = (String) json.get("UserId");

                QuestionDAO dao = new QuestionDAO();
                dao.createQuestion(id, subject, content, (int) category, userId, gc.getTimeInMillis());
            } catch (Exception ex) {
                System.out.println("K ==> " + k);
            }
        }
        System.out.println("Finish!");
    }

    public static void createUser() {
        String f = "dataset/yh_computer_internet_q";
        JSONParser parser = new JSONParser();
        System.out.println("Running....");
        UserProfileDAO dao = new UserProfileDAO();

        for (int k = 1002; k < 2050; ++k) {
            try {
                String fname = f + k;
                //System.out.println("FILE: " + fname);
                JSONObject jsonRoot = (JSONObject) parser.parse(new BufferedReader(new FileReader(fname)));

                JSONObject jsonQuestions = (JSONObject) jsonRoot.get("all");

                JSONArray jsonArrQuestions = (JSONArray) jsonQuestions.get("question");
                JSONObject json = (JSONObject) jsonArrQuestions.get(0);

                //JSONObject answers = (JSONObject) jsonArrQuestions.get(1);
                JSONArray arrAnswers = (JSONArray) jsonQuestions.get("answers");

                String userid = (String) json.get("UserId");
                String usernick = (String) json.get("UserNick");
                dao.createUserProfile(new UserProfile(userid, usernick));
                for (int i = 0; i < arrAnswers.size(); ++i) {
                    JSONObject obj = (JSONObject) arrAnswers.get(i);
                    userid = (String) obj.get("UserId");
                    usernick = (String) obj.get("UserNick");
                    dao.createUserProfile(new UserProfile(userid, usernick));
                }

            } catch (Exception ex) {
                ex.printStackTrace();
                System.out.println("K ==> " + k);
            }
        }
        System.out.println("Finish!");
    }

    public static void createAnswers() {
        String f = "dataset/yh_computer_internet_q";
        JSONParser parser = new JSONParser();
        System.out.println("Running....");
        AnswerDAO dao = new AnswerDAO();
        SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        for (int k = 0; k < 2005; ++k) {
            try {
                String fname = f + k;
                //System.out.println("FILE: " + fname);
                JSONObject jsonRoot = (JSONObject) parser.parse(new BufferedReader(new FileReader(fname)));

                JSONObject jsonQuestions = (JSONObject) jsonRoot.get("all");

                JSONArray jsonArrQuestions = (JSONArray) jsonQuestions.get("question");
                JSONObject json = (JSONObject) jsonArrQuestions.get(0);

                String qid = (String) json.get("Id");
                String _qid = qid.substring(14);
                //JSONObject answers = (JSONObject) jsonArrQuestions.get(1);
                JSONArray arrAnswers = (JSONArray) jsonQuestions.get("answers");

                for (int i = 0; i < arrAnswers.size(); ++i) {
                    JSONObject obj = (JSONObject) arrAnswers.get(i);
                    String userid = (String) obj.get("UserId");
                    String date = (String) obj.get("Date");
                    Date d = df.parse(date);
                    Calendar gc = new GregorianCalendar();
                    gc.setTime(d);

                    date = date.replaceAll(" ", "");
                    date = date.replaceAll("-", "");
                    date = date.replaceAll(":", "");
                    String id = date + userid + _qid;
                    String content = (String) obj.get("Content");

                    String best = (String) obj.get("Best");
                    if (best.isEmpty()) {
                        best = "0";
                    }
                    int rate = Integer.parseInt(best);
                    dao.updateRate(id, rate); //.createAnswer(id, content, qid, userid, gc.getTimeInMillis());
                }

            } catch (Exception ex) {
                ex.printStackTrace();
                System.out.println("K ==> " + k);
            }
        }
        System.out.println("Finish!");
    }
    
    public static void createFeedback() {
        String f = "dataset/yh_computer_internet_q";
        JSONParser parser = new JSONParser();
        System.out.println("Running....");
        FeedbackDAO dao = new FeedbackDAO();
        SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        Random rnd = new Random();
        for (int k = 0; k < 2005; ++k) {
            try {
                String fname = f + k;
                //System.out.println("FILE: " + fname);
                JSONObject jsonRoot = (JSONObject) parser.parse(new BufferedReader(new FileReader(fname)));

                JSONObject jsonQuestions = (JSONObject) jsonRoot.get("all");

                JSONArray jsonArrQuestions = (JSONArray) jsonQuestions.get("question");
                JSONObject json = (JSONObject) jsonArrQuestions.get(0);

                String qid = (String) json.get("Id");
                String _qid = qid.substring(14);
                String quid = (String) json.get("UserId");
                //JSONObject answers = (JSONObject) jsonArrQuestions.get(1);
                JSONArray arrAnswers = (JSONArray) jsonQuestions.get("answers");
                
                boolean ok = check(qid);
                
                String auid = (String) json.get("ChosenAnswererId");

                for (int i = 0; i < arrAnswers.size(); ++i) {
                    JSONObject obj = (JSONObject) arrAnswers.get(i);
                    String userid = (String) obj.get("UserId");
                    String date = (String) obj.get("Date");
                    Date d = df.parse(date);
                    Calendar gc = new GregorianCalendar();
                    gc.setTime(d);

                    date = date.replaceAll(" ", "");
                    date = date.replaceAll("-", "");
                    date = date.replaceAll(":", "");
                    String id = date + userid + _qid;
                    String content = (String) obj.get("Content");

                    String best = (String) obj.get("Best");
                    if (best.isEmpty()) {
                        best = "0";
                    }
                    int rate = Integer.parseInt(best);
                    
                    // Best question
                    if(auid.equals(userid)) {
                        if(ok) {
                            dao.createFeedback(quid, userid, id, 1, gc.getTimeInMillis());
                        }
                    } else if(rate == 0){
                        boolean untrust = rnd.nextBoolean();
                        if(untrust) {
                            dao.createFeedback(quid, userid, id, 0, gc.getTimeInMillis());
                        }
                    }
                    
                }

            } catch (Exception ex) {
                ex.printStackTrace();
                System.out.println("K ==> " + k);
            }
        }
        System.out.println("Finish!");
    }

   
}
